package util.cleaner;

import java.util.HashSet;
import java.util.Hashtable;

import udf.contants.Contants;
import util.io.FileInput;

public class TrimByVocabulary {


	static private int _id_url = 0;
	static private int _id_tag = 1;
	static private int _url = 2;
	static private int _tag = 3;
	static private int _freq_u = 5;
	static private int _freq_tag = 4;
	static private int _freq_ut = 6;
	static private int _age = 7;
	
	private static int _adult_freq_tag= 8;
	private static int _adult_freq_url=9;
	private static int _adult_freq_domain=10;
	
	int offset= 10000000;
	
	
	static private int gold_query=6;
	static private int gold_reformulation=7;
	static private int gold_age =4;
	private static HashSet<String> stopWords;
	

	
	public static  void initStopWords()  {
		stopWords= new HashSet<String>();
		for (int i = 0; i < udf.contants.Contants.stop_words_simple.length; i++) {

			stopWords.add(Contants.stop_words_simple[i].trim().toLowerCase());

		}

	}

	
	public static Hashtable<String,Integer> getFreq(String path){
		Hashtable<String,Integer> freq= new Hashtable<String,Integer>();
		initStopWords() ;
		FileInput in = new FileInput(path);
		String line = in.readString();
		
		while(line!=null){
			String t[]= line.split("\t");
			//System.out.println(line);
			String query = t[gold_query];
			String refo = t[gold_reformulation];
			String age = t[gold_age];
			
			
		
			if(gold_reformulation>t.length || age.contains("adults") || age.equals("years_16_19") ){
				line= in.readString();
				continue;
				
			}
			
		
			
			
			
			
			String q[] = query.split("\\s+");
			String r []= refo.split("\\s+");
			
			for(int i =0;   i <q.length; i++){
				
				
				if(!stopWords.contains(q[i])  ){
					if(freq.containsKey(q[i])){
						
						
						freq.put(q[i], freq.get(q[i])+1);
					}else{
						
						freq.put(q[i],1);
					}
					
				}
			}
				
				
			for(int i =0; i <r.length; i++){
				
				if(!stopWords.contains(r[i])){
					if(freq.containsKey(r[i])){
						
						freq.put(r[i], freq.get(r[i])+1);
					}else{
						
						freq.put(r[i], 1);
					}
					
				}
			}
			line = in.readString();
			
			}
		return freq;
		}
		
		
	public static boolean hasTokenInDict(String tag, Hashtable<String,Integer> freq){
		
		
		
		String t[]= tag.split("_");
	
		for(int i =0; i<t.length; i++){
			
			
			if(!freq.containsKey(t[i]) ){
				
				return false;
			}
		}
	
	return true;
	}
		
	
	public static boolean isInvalidTag(String tag){
		
		if(tag.length()<2) return true;
		
		if(tag.matches("[0-9]_[0-9]")) return true;
		
		String t[]= tag.split("_");
		for(int i=0; i<t.length; i++){
			
			if(t[i].length()<2){
				return true;
			}
		}
		
		
		
		return false;
	}
	
	public static void trimByFreq(String path, String path_gold){
		
		Hashtable<String, Integer> gold = getFreq(path_gold);
		
	
		FileInput in = new FileInput(path);
		String line = in.readString();
		int ids = 0;
		if (gold.containsKey("3w")){
			
			System.out.println("WTF....");
			System.exit(0);
		}
		

		Hashtable<String, Integer> urls_h = new Hashtable<String, Integer>();

		Hashtable<String, Integer> tags_h = new Hashtable<String, Integer>();

		// add url ids

		while (line != null) {

			String[] fields = line.split("\t");

			Integer id_url = Integer.valueOf(Integer.valueOf(fields[_id_url]));
			Integer id_tag = Integer.valueOf(Integer.valueOf(fields[_id_tag]));

			Integer freq_tag = Integer.valueOf(Integer
					.valueOf(fields[_freq_tag]));
			Integer freq_url = Integer
					.valueOf(Integer.valueOf(fields[_freq_u]));
			Integer freq_ut = Integer
					.valueOf(Integer.valueOf(fields[_freq_ut]));
			String age = fields[_age];
			String name = fields[_tag];
			String url_name = fields[_url];

			if(isInvalidTag(name) || InvalidPatterns.matchInvalidPattern(name)){
				line = in.readString();
				continue;
			}
			
			
			if(hasTokenInDict(name,gold) || (gold.containsKey(name)  && !stopWords.contains(name))){
				
			
				if(hasTokenInDict(name,gold)){
					
					//System.out.println(name );
					
				}
				if(!age.contains("adults")){
					urls_h.put(url_name, id_url);
					tags_h.put(name, id_tag);
					
				}
				
			}

			line = in.readString();
		}
		
		
		//regerate file
		
		in = new FileInput(path);
		line= in.readString();
	
		while(line!=null){

			String[] fields = line.split("\t");

			Integer id_url = Integer.valueOf(Integer.valueOf(fields[_id_url]));
			Integer id_tag = Integer.valueOf(Integer.valueOf(fields[_id_tag]));

			Integer freq_tag = Integer.valueOf(Integer.valueOf(fields[_freq_tag]));
			Integer freq_url = Integer.valueOf(Integer.valueOf(fields[_freq_u]));
			Integer freq_ut = Integer.valueOf(Integer.valueOf(fields[_freq_ut]));
			String age = fields[_age];
			String name = fields[_tag];
			String url_name = fields[_url];
			
			Integer adult_freq_tag =0;
			Integer adult_freq_url=0;
			Integer adult_freq_domain=0;
			
			if(fields.length>_adult_freq_domain){
			 adult_freq_tag =  Integer.valueOf(Integer.valueOf(fields[_adult_freq_tag]));
			
			 adult_freq_url =  Integer.valueOf(Integer.valueOf(fields[_adult_freq_url]));
			
			 adult_freq_domain =  Integer.valueOf(Integer.valueOf(fields[_adult_freq_domain]));
			
			}

			if(urls_h.containsKey(url_name) && tags_h.containsKey(name))
			{
				
				if(fields.length<=_adult_freq_domain){
					
					System.out.println(id_url + "\t"+ id_tag + "\t"+ url_name + "\t"+ name +"\t"
							+freq_tag + "\t"+ freq_url + "\t"+ freq_ut + "\t"+ age);
				}else{
					System.out.println(id_url + "\t"+ id_tag + "\t"+ url_name + "\t"+ name +"\t"
							+freq_tag + "\t"+ freq_url + "\t"+ freq_ut + "\t"+ age + "\t"+ adult_freq_tag + "\t"+ adult_freq_url + "\t"+ adult_freq_domain);
				}
				
			
			
			}
			line=in.readString();
			
		}
	
	}
	
	
	
	
	public static void main(String[] args) {
		// TODO Auto-generated method stub

		String path_gold ="/Users/sergioduarte/projects/data/query_suggestions/query_suggestions/query_suggestions.txt";
		String prefix = "/Users/sergioduarte/projects/data/graph/";
		
		//String path = "/media/sata__/data/delicious_graph/delicious_graphML_raw_ids_dmoz.txt";
		String path = prefix+ "delicious_graphML_raw_ids_dmoz_youth_mix.txt";
		
		trimByFreq(path,path_gold);
		
		
		
	}

}
